{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*************************************************\n",
      "Grapping comment data of app:137520 from TapTap...\n",
      "*************************************************\n",
      "\n",
      "Page 1 grapped,  0.96 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 2 grapped,  5.92 seconds used\n",
      "Page 3 grapped,  1.00 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 4 grapped,  6.02 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 5 grapped,  6.05 seconds used\n",
      "Page 6 grapped,  1.07 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 7 grapped,  6.20 seconds used\n",
      "Page 8 grapped,  1.12 seconds used\n",
      "Page 9 grapped,  1.07 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 10 grapped,  6.04 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 11 grapped,  6.01 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 12 grapped,  5.98 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 13 grapped,  5.98 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 14 grapped,  5.95 seconds used\n",
      "Page 15 grapped,  0.91 seconds used\n",
      "Page 16 grapped,  1.10 seconds used\n",
      "Page 17 grapped,  1.05 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 18 grapped,  5.93 seconds used\n",
      "Page 19 grapped,  1.20 seconds used\n",
      "Page 20 grapped,  0.96 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 21 grapped,  5.96 seconds used\n",
      "Page 22 grapped,  1.05 seconds used\n",
      "Page 23 grapped,  1.03 seconds used\n",
      "Page 24 grapped,  0.98 seconds used\n",
      "连接超时，第1次重试...\n",
      "Page 25 grapped,  5.90 seconds used\n",
      "\n",
      "*************************************************\n",
      "Comment grapping finished. 500 comments grapped in total\n",
      "Written to 【tap_comment_appid137520.xlsx】\n",
      "*************************************************\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "import time as tm\n",
    "\n",
    "#************************************\n",
    "#功能：发起HTTP请求，获取页面h5文本并通过BeautifulSoup解析\n",
    "#参数：url：需要抓取的页面地址\n",
    "#输出：page_content_bs：加载好的美味汤对象\n",
    "#************************************\n",
    "def url_to_bs(url):\n",
    "    headers = {\n",
    "        'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',\n",
    "        'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36',\n",
    "    }\n",
    "    retry_count = 0\n",
    "    while retry_count < 5:\n",
    "        try:\n",
    "            page_content = requests.get(url,headers=headers,timeout=5)  #加载url\n",
    "            page_content_bs = BeautifulSoup(page_content.text, \"html.parser\")  #把url对象转化为美味汤\n",
    "            return page_content_bs\n",
    "    \n",
    "        except requests.exceptions.RequestException:\n",
    "            retry_count += 1\n",
    "            print('连接超时，第{}次重试...'.format(retry_count))\n",
    "\n",
    "#************************************\n",
    "#功能：以下3个函数都是一样的原理，通过bs和re,使用正则表达式获取所需的内容\n",
    "#参数：bs：加载好的美味汤对象\n",
    "#输出：pinglun：评论文本；score_num：评论分数（5分制）；datetime：评论时间\n",
    "#************************************\n",
    "def get_comment_text(bs):\n",
    "    comment = bs.select(\".review-item-text \")                         #选择页面中的评论模块内容\n",
    "    pattern_pinglun = '<div class.*?data-review.*?\"contents\">(.*?)</div>'  #构建评论的正则表达，选取(.*?)中的内容\n",
    "    pinglun = re.findall(pattern_pinglun, str(comment), re.S)              #根据选中模块和正则，抓取需要的内容 \n",
    "    return pinglun\n",
    "\n",
    "def get_comment_score(bs):\n",
    "    comment = bs.select(\".review-item-text \")                         #选择页面中的评论模块内容\n",
    "    pattern_score='<i class.*?\"width: (.*?)px\"></i>'                       #构建评分的正则表达，选取(.*?)中的内容\n",
    "    score = re.findall(pattern_score, str(comment), re.S)                  #根据选中模块和正则，抓取需要的内容 \n",
    "    score_num = [ int(x)/14 for x in score ]                               #抓出来的是字符串（像素宽度），转化为整型，并计算评分    \n",
    "    return score_num\n",
    "\n",
    "def get_comment_datetime(bs):\n",
    "    comment_header = bs.select(\".review-item-text .item-text-header\") #选择页面中评论模块的头部\n",
    "    pattern_datetime='data-dynamic-time=\".*?\">(.*?)</span>'                    #构建评论日期的正则表达，选取(.*?)中的内容\n",
    "    datetime=re.findall(pattern_datetime, str(comment_header), re.S)               #根据选中模块和正则，抓取需要的内容    \n",
    "    return datetime\n",
    "\n",
    "\n",
    "#step1 准备要爬取的产品的信息\n",
    "app_id=\"137520\"      #需要爬取的游戏的tap里面的id，可以在产品页的Url找到\n",
    "page_total=25       #需要爬取的总页数\n",
    "\n",
    "#准备输出容器\n",
    "comment_out=[]\n",
    "score_out=[]\n",
    "datetime_out=[]\n",
    "\n",
    "print('*************************************************')\n",
    "print('Grapping comment data of app:{0} from TapTap...'.format(app_id))\n",
    "print('*************************************************\\n')\n",
    "\n",
    "#step2 由于需要爬去多页数据，建立循环爬取机制\n",
    "for j in range(1,page_total+1):\n",
    "    t1=tm.time()\n",
    "    link=\"https://www.taptap.com/app/{0}/review?order=update&page={1}#review-list\".format(app_id, j)   #拼接每一页的url\n",
    "    \n",
    "#step3 抓取单页数据\n",
    "    star_bs=url_to_bs(link)                            #加载BeautifulSoup\n",
    "    \n",
    "    comment_tmp=get_comment_text(star_bs)              #获取评论\n",
    "    score_tmp=get_comment_score(star_bs)               #获取分数\n",
    "    datetime_tmp=get_comment_datetime(star_bs)                 #获取评论时间\n",
    "    \n",
    "    comment_out.extend(comment_tmp)                    #装入输出容器\n",
    "    score_out.extend(score_tmp)\n",
    "    datetime_out.extend(datetime_tmp)\n",
    "    t2=tm.time()\n",
    "    timing=t2-t1                                       #计时，用于调试\n",
    "    print('Page %d grapped, %5.2f seconds used' % (j,timing))  #输出爬取进度\n",
    "\n",
    "#step4 整理成数据框格式，导出数据\n",
    "result={\"comment\" : comment_out,\n",
    "        \"score\" : score_out,\n",
    "        \"comment_date\" : datetime_out}                     #先把列表转为字典\n",
    "\n",
    "resultpd=pd.DataFrame(result)                          #再把字典转为pandas数据框  \n",
    "resultpd['comment']=resultpd['comment'].str.replace(\"\\n<p>\",\"\").replace(\"</p><p>\",\" \")\n",
    "\n",
    "print('\\n*************************************************')\n",
    "print('Comment grapping finished. %d comments grapped in total' % (len(comment_out)))\n",
    "resultpd.to_excel('tap_comment_appid{}.xlsx'.format(app_id))\n",
    "print('Written to 【tap_comment_appid{}.xlsx】'.format(app_id))\n",
    "print('*************************************************')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'resultpd' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-ac33da1de85c>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mresultpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'resultpd' is not defined"
     ]
    }
   ],
   "source": [
    "resultpd.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
